#########################################################################
# Time-stamp: <05/09/09 11:06:23 ostolop>
#########################################################################
package EP::EPC::Core::Clustering;

use strict;

use EP::Config;
use EP::Common;
use EP::Common::General;

use EP::EPC::Core;
use EP::EPC::Core::Component;
use EP::EPC::Core::Clustering::exp;

use EP::Visualization::Grid;
use EP::Visualization::Hooks;
use EP::Visualization::HeatmapGraphic;
use EP::Visualization::LineGraphic;
use EP::Visualization::TreeGraphic;
use EP::Visualization::TreemapGraphic;
use EP::Visualization::AnnotationGraphic;
use EP::Visualization::Collapser;

use PDL;
use PDL::NiceSlice;

our @ISA = qw ( EP::EPC::Core::Component );

# TODO: REFACTOR THIS WHOLE MODULE

sub process_component {
  my $self = shift;
  $self->{query} = shift if scalar @_ > 0;

  my ($vis_file, $width, $height) = $self->process_clustering ( $self->{src_dataset} );

  my $process_result;

  if ( $vis_file =~ /svg$/ ) {
    $process_result = {
           visualization  => { type  => "image/svg+xml",
             label => "Clustering Visualization",
             value => "$EP::Config::h->{HTML_HREF}/$EP::Config::h->{WWW_TMP}/$vis_file" },
           visualization_width => $width,
           visualization_height => $height,
           src_dataset_id => $self->{src_dataset}->id,
           dst_dataset_id => $self->{src_dataset}->id
          };
  } else {
    $process_result = {
           visualization  => { type  => "image/png",
             label => "Clustering Visualization",
             value => "$EP::Config::h->{HTML_HREF}/$EP::Config::h->{WWW_TMP}/$vis_file" },
           visualization_image_map => "$EP::Config::h->{HTML_HREF}/$EP::Config::h->{WWW_TMP}/$vis_file.map",
           src_dataset_id => $self->{src_dataset}->id,
           dst_dataset_id => $self->{src_dataset}->id
          };
  }

  foreach my $metadata_name ( keys %{$self->{src_dataset}->{action}->{action_metadata}} ) {
    next if $metadata_name eq "heatmap_data" or $metadata_name eq "rows" or $metadata_name eq "cols";
    $process_result->{$metadata_name} = $self->{src_dataset}->{action}->{action_metadata}->{$metadata_name};
  }

  delete $self->{src_dataset}->{action};
  $self->{CD}->{process_result} = $process_result;

  return $self->record_analysis_history ();
}

sub process_clustering {
  my $self = shift;

  my $svg_file;
  my $width;
  my $height;

  $self->{session}->status ( "Processing ..." );

  $_ = $self->{src_dataset}->type;

 SWITCH: {
    /exp/
      && do {
#   if ( $self->{query}->param("cluster_vis") == 1 and $self->{query}->param("cluster_what") == 3 ) {
#     die "A treemap displays clustering by rows or colums, not by both!\n";
#     last SWITCH;
#   }

  EP::EPC::Core::Clustering::exp::process_clustering ( $self, $self->{src_dataset}, $self->{query} );
  ( $svg_file, $width, $height ) = $self->create_visualization ();
  last SWITCH;
      };
    /seq/
      && do {
  last SWITCH;
      };

    /affy/
      && do {
  throw EP::Common::Error ( -value => AFFY_TRANSFORM_RAW_DATA );
  last SWITCH;
      };
  }

  return ($svg_file, $width, $height);
}

sub get_action_metadatax {
  my $self = shift;

  $self->{session}->status ( "Recording analysis history ..." );

  my $action_metadata = $self->{src_dataset}->{action}->{action_metadata};

  if ( ref $action_metadata eq "HASH" ) {
    delete $action_metadata->{heatmap_data} if exists $action_metadata->{heatmap_data};
    delete $action_metadata->{rows}         if exists $action_metadata->{rows};
    delete $action_metadata->{cols}         if exists $action_metadata->{cols};
  }

  return $action_metadata;
}

sub create_visualization {
  my $self = shift;

  $self->{session}->status ( "Running visualization ..." );

  #  my $action_id = $self->{query}->param ( "action_id" );
  #  my $action = EP::Common::DB::get_action ( { id => $action_id } );

  my $dataset = $self->{src_dataset};
  my $action = $dataset->{action};

  # check if this action is visualizable (?????) <<== interfaces!
  die "Action not visualizable" if ( not $action->{visualizable} );
  my $grid;

  my $override_vis_format_to_png = 0;

  # test what kind of action is this (?????)
  # test comp_id == 3 for hierarchical clustering
  if ( $action->{component_id} == 3 ) {
    my @v_hooks = sort { $a <=> $b} keys %{$action->{action_metadata}->{rows}};
    my @h_hooks = sort { $a <=> $b} keys %{$action->{action_metadata}->{cols}};

    # initialize cell sizes to small
    my ($cell_height, $cell_width ) = ( 1, 1 );

    if ( scalar @v_hooks < 200 ) {
      $cell_height = 12;
    } elsif ( scalar @v_hooks < 2000 ) {
      $cell_height = 12;
    }

    if ( scalar @h_hooks < 200 ) {
      $cell_width = 12;
    } elsif ( scalar @h_hooks < 2000 ) {
      $cell_width = 12;
    }

    my $V_hook = new EP::Visualization::Hooks ( \@v_hooks );
    my $H_hook = new EP::Visualization::Hooks ( \@h_hooks );

    #create heatmap
    my $heatmap = new EP::Visualization::HeatmapGraphic ( data => $action->{action_metadata}->{heatmap_data},
                cell_height => $cell_height, cell_width => $cell_width );
    $heatmap->setVerticalHookSet   ( $V_hook );
    $heatmap->setHorizontalHookSet ( $H_hook );

    #create annotation
    my $row_annotate = $action->{action_metadata}->{rows};
    my @row_annots;
    push @row_annots, $row_annotate->{$_} foreach sort { $a <=> $b } keys %{$row_annotate};

    my $row_annotation = new EP::Visualization::AnnotationGraphic ( data => \@row_annots, 'font-size' => $cell_height, 'no-color-efs' => 1 );
    $row_annotation->setHook ( $V_hook );

    my $col_annotate = $action->{action_metadata}->{cols};
    my @col_annots;
    push @col_annots, $col_annotate->{$_} foreach sort { $a <=> $b } keys %{$col_annotate};

    my $col_annotation = new EP::Visualization::AnnotationGraphic ( data => \@col_annots, 'font-size' => $cell_width );
    $col_annotation->setHook ( $H_hook );
    $col_annotation->setOrientation(-90);

    #####################
    # IF IT'S A TREEMAP
    #####################

    if ($self->{query}->param("cluster_vis") == 1){


        # TODO: and if "not exists" ??
        warn "We are now reading trees in....";
        my $treemap;
        SWITCH: for ($self->{query}->param("cluster_what")) {
          /0/     && do {
            if (exists $action->{action_metadata}->{rows_clust}){
              $treemap = new EP::Visualization::TreemapGraphic (
              data => "$EP::Config::h->{EPNG_USRDATA_PATH}/" . $dataset->folder->location . "/$action->{action_metadata}->{rows_clust}",
              heatmap_data => $action->{action_metadata}->{heatmap_data},
              rows => $action->{action_metadata}->{rows},
              species => $dataset->species(),
	      dataset => $dataset,
	      root_dataset => $self->{session}->current_root_dataset );
            }
            else {
              warn "no rows_clust in action_metadata";
            }
            last;
          };
          /1/          && do {
            if (exists $action->{action_metadata}->{rows_clust}){
              $treemap = new EP::Visualization::TreemapGraphic ( data => "$EP::Config::h->{EPNG_USRDATA_PATH}/" . $dataset->folder->location . "/$action->{action_metadata}->{cols_clust}", heatmap_data => $action->{action_metadata}->{heatmap_data},
              rows => $action->{action_metadata}->{rows},
              species => $dataset->species(),
              dataset => $dataset,
	      root_dataset => $self->{session}->current_root_dataset  );
            }
            else {
              warn "no rows_clust in action_metadata";
            }
            last;
          };
          /3/        && do {
            #die "A Treemap displays clustering by rows OR colums; not by both\n";
            if (exists $action->{action_metadata}->{rows_clust}){
              $treemap = new EP::Visualization::TreemapGraphic ( data => "$EP::Config::h->{EPNG_USRDATA_PATH}/" . $dataset->folder->location . "/$action->{action_metadata}->{rows_clust}", heatmap_data => $action->{action_metadata}->{heatmap_data},
              rows => $action->{action_metadata}->{rows},
              species => $dataset->species(),
              dataset => $dataset,
	      root_dataset => $self->{session}->current_root_dataset );
            }
            else {
              warn "no rows_clust in action_metadata";
            }
            last;
          };
          die "unknown value for form variable: ".$self->{query}->param("cluster_what");
        }
        $grid = new EP::Visualization::Grid ( 3, 3 );

        my $treemap_panel = $treemap->get_control_panel();
        $grid->fillCellArray (
            [
              [ $treemap, $treemap_panel ],
            ]
        );

        $action->{action_metadata}->{rows_clust} = { value => $action->{action_metadata}->{rows_clust}, uuid => $treemap->get_uuid() } if $self->{query}->param ( "cluster_what" ) == 0;
        $action->{action_metadata}->{cols_clust} = { value => $action->{action_metadata}->{cols_clust}, uuid => $treemap->get_uuid() } if $self->{query}->param ( "cluster_what" ) == 1;
        $action->{action_metadata}->{cols_clust} = { value => $action->{action_metadata}->{cols_clust}, uuid => $treemap->get_uuid() } if $self->{query}->param ( "cluster_what" ) == 3;
    }
    else {
        # TODO: and if "not exists" ??
        warn "We are now reading trees in....";
        my $rows_tree = new EP::Visualization::TreeGraphic ( data => "$EP::Config::h->{EPNG_USRDATA_PATH}/" . $dataset->folder->location . "/$action->{action_metadata}->{rows_clust}",
                  leaf_height => $cell_height
                      ) if exists $action->{action_metadata}->{rows_clust} and ( $self->{query}->param("cluster_what") != 1 );
        my $cols_tree = new EP::Visualization::TreeGraphic ( data => "$EP::Config::h->{EPNG_USRDATA_PATH}/" . $dataset->folder->location . "/$action->{action_metadata}->{cols_clust}",
                  leaf_height => $cell_width
                      ) if exists $action->{action_metadata}->{cols_clust} and ( $self->{query}->param("cluster_what") != 0 );

        # Check if trees should be collapsed and collapse them
        if ( $self->{query}->param ( "COLLAPSECRITERIA_GENES" ) eq "percent" && $self->{query}->param ( "COLLAPSECRITERIA_CONDITIONS" ) eq "percent") {
          my $collapser = new EP::Visualization::Collapser();
          $collapser->collapse ( $self->{query}->param ( "COLLAPSE_GENES"      ), "row",    $heatmap, $rows_tree, $row_annotation );
          $collapser->collapse ( $self->{query}->param ( "COLLAPSE_CONDITIONS" ), "column", $heatmap, $cols_tree, $col_annotation );
        } elsif ( $self->{query}->param ( "COLLAPSECRITERIA_GENES" ) eq "percent" ) {
          my $collapser = new EP::Visualization::Collapser();
          $collapser->collapse ( $self->{query}->param ( "COLLAPSE_GENES" ),      "row",    $heatmap, $rows_tree, $row_annotation );
        } elsif ( $self->{query}->param ( "COLLAPSECRITERIA_CONDITIONS" ) eq "percent" ) {
          my $collapser = new EP::Visualization::Collapser();
          $collapser->collapse ( $self->{query}->param ( "COLLAPSE_CONDITIONS" ), "column", $heatmap, $cols_tree, $col_annotation );
        }

        #set the ordering of heatmap and annotation by the trees
        $heatmap->orderHorizontalBy ( $cols_tree ) if $cols_tree;
        $heatmap->orderVerticalBy   ( $rows_tree ) if $rows_tree;
        $col_annotation->orderBy    ( $cols_tree ) if $cols_tree;
        $row_annotation->orderBy    ( $rows_tree ) if $rows_tree;

        # if too much data (i.e. cell sizes remain tiny), do not display annotations.
        $row_annotation = undef if $cell_height == 1;
        $col_annotation = undef if $cell_width == 1;

        $grid = new EP::Visualization::Grid ( 3, 3 );

        # add graphics objects to the grid according to selected layout
        $rows_tree->setOrientation ( 90 ) if $rows_tree; # the rows tree should be rotated to display correctly as the side of the heatmap

        $grid->fillCellArray ( [
              [ undef,           $cols_tree,      undef      ],
              [ undef,           $col_annotation, undef      ],
              [ $row_annotation, $heatmap,        $rows_tree ],
            ]
          );

        $action->{action_metadata}->{rows_clust} = { value => $action->{action_metadata}->{rows_clust}, uuid => $rows_tree->get_uuid() } if $rows_tree;
        $action->{action_metadata}->{cols_clust} = { value => $action->{action_metadata}->{cols_clust}, uuid => $cols_tree->get_uuid() } if $cols_tree;
    }
  ###############################################
  } elsif ( $action->{component_id} == 4 or $action->{component_id} == 13 ) {
    # if merging all clusters into 1, reset the K.
    $action->{action_metadata}->{K} = 1 if $action->{action_metadata}->{merge_sim_results} == 1;

    $grid = new EP::Visualization::Grid ( 6, 3 * $action->{action_metadata}->{K} );

    open CLUST, "$EP::Config::h->{EPNG_USRDATA_PATH}/" . $dataset->folder->location . "/$action->{action_metadata}->{clustering}"
      or die "Couldn't open clustering results file $action->{action_metadata}->{clustering}!";

    my @cell_array;
    my %k_means_members;

    while (<CLUST>) {
      next unless /CLUSTER \d+/;
      my ($cluster_nr, $cluster_size) = $_ =~ /CLUSTER (\d+?) SIZE=(\d+?)/;

      # cluster members
      $_ = <CLUST>;
      my @members = split /\s/;
      shift @members;
      $_-- foreach @members;

      next if @members == 0;

      # cluster centers
      <CLUST>;
      my @centers = split /\s/;
      shift @centers;

      $k_means_members{$cluster_nr}->{members} = \@members unless $action->{action_metadata}->{merge_sim_results} == 1;

      if ( $action->{action_metadata}->{merge_sim_results} == 1 ) {
  $k_means_members{1} = {} if not exists $k_means_members{1};
  $k_means_members{1}->{$_} = 1 foreach @members;
      } else {
  $k_means_members{$cluster_nr} = {};
  $k_means_members{$cluster_nr}->{$_} = 1 foreach @members;
      }
    }

    foreach my $cluster_nr ( keys %k_means_members ) {
      my @members = keys %{$k_means_members{$cluster_nr}};

      my $d = $action->{action_metadata}->{heatmap_data};
      my $heatmap_data    = $d(:, pdl @members);

      my @heatmap_H_hooks = sort { $a <=> $b} keys %{$action->{action_metadata}->{cols}};

      my $V_hook = new EP::Visualization::Hooks ( \@members, $action->{action_metadata}->{rows} );
      my $H_hook = new EP::Visualization::Hooks ( \@heatmap_H_hooks, $action->{action_metadata}->{cols} );

      #create annotation
      my $row_annotate = $action->{action_metadata}->{rows};
      my @row_annots;
      push @row_annots, $row_annotate->{$_} foreach @members;

      # initialize cell sizes to small
      my ($cell_height, $cell_width ) = ( 1, 1 );

      if ( scalar @members < 200 ) {
  $cell_height = 12;
      } elsif ( scalar @members < 2000 ) {
  $cell_height = 12;
      }

      if ( scalar @heatmap_H_hooks < 200 ) {
  $cell_width = 12;
      } elsif ( scalar @heatmap_H_hooks < 2000 ) {
  $cell_width = 12;
      }

      my $row_annotation = new EP::Visualization::AnnotationGraphic ( data => \@row_annots, 'font-size' => $cell_height, 'no-color-efs' => 1 );
      $row_annotation->setHook ( $V_hook );

      my $col_annotate = $action->{action_metadata}->{cols};
      my @col_annots;
      push @col_annots, $col_annotate->{$_} foreach sort { $a <=> $b} keys %{$col_annotate};

      my $col_annotation = new EP::Visualization::AnnotationGraphic ( data => \@col_annots, 'font-size' => $cell_width );
      $col_annotation->setHook ( $H_hook );
      $col_annotation->setOrientation(-90);

      my $cluster_heatmap  = new EP::Visualization::HeatmapGraphic ( data => $heatmap_data, cell_height => $cell_height, cell_width => $cell_width, interactive => 0 );
      $cluster_heatmap->setVerticalHookSet   ( $V_hook );
      $cluster_heatmap->setHorizontalHookSet ( $H_hook );

      my $cluster_lineplot = new EP::Visualization::LineGraphic ( data => $heatmap_data, plot_height => 150, cell_width => $cell_width, interactive => 0 );
# TODO: (commenting out interactivity) restore passing params from result display:    interactive => ( $action->{component_id} == 13 ? 1 : 0 ) );
      $cluster_lineplot->setVerticalHookSet   ( $V_hook );
      $cluster_lineplot->setHorizontalHookSet ( $H_hook );

      my $grid_row = 3 * ( $cluster_nr - 1 );

      push @cell_array, (
       [ undef,           undef, $col_annotation,  undef, $col_annotation   ],
       [ $row_annotation, undef, $cluster_heatmap, undef, $cluster_lineplot ],
       [ undef,           undef, undef,            undef, undef             ]
      );

      my $kthclustfile = "$EP::Config::h->{EPNG_USRDATA_PATH}/" . $self->{src_dataset}->folder->location . "/$action->{action_metadata}->{clustering}.k$cluster_nr.members";
      open KTHCLUST, ">$kthclustfile"  or die "Can't open for writing $cluster_nr -th cluster members file $kthclustfile";
      print KTHCLUST "# CLUSTER MEMBERS: ", join ( ",", @members ), "\n";
      print KTHCLUST join ( "\t", $row_annots[0]->{full_annot}->Keys ), "\n";
      foreach ( @row_annots ) {
	print KTHCLUST join ( "\t", $_->{full_annot}->Values ), "\n";
      }
      close KTHCLUST;

      $action->{action_metadata}->{"clustering_$cluster_nr"} = { type => "text/plain",
								 label => "Cluster $cluster_nr",
								 value => "$EP::Config::h->{EPNG_USRDATA_URL}/" . $self->{src_dataset}->folder->location . "/$action->{action_metadata}->{clustering}.k$cluster_nr.members"
							       };
    }

    close CLUST;
    $grid->fillCellArray ( \@cell_array );
  }

  # always override to PNG if not drawing treemap
  if ( $self->{query}->param("cluster_vis") != 1 ) {
    $override_vis_format_to_png = 1;# if ( $action->{action_metadata}->{heatmap_data}->dim(1) > 300 );
    $override_vis_format_to_png = 1;# if ( $action->{action_metadata}->{heatmap_data}->dim(0) > 50 );
  }

  die "Failed to generate visualization!" if not defined $grid;

  my $vis_format = $self->{current_user}->visualization_format;

  $vis_format = 1 if $override_vis_format_to_png;
  $vis_format = 0 if $self->{query}->param("cluster_vis") == 1;

  if ( $self->{current_user}->visualization_format == 1 and $self->{query}->param ( "cluster_vis" ) == 1 ) {
    die "Treemap visualization is only available in SVG.\nPlease make sure an SVG viewer on your system is installed\n and change your EP:NG user preferences accordingly.";
  }

  my $r = rand(1000);
  my $svg_output = "vis.$dataset.$$.$r.svg";
  my $vis_output_filename = "vis.$dataset.$$.$r." . ( $vis_format == 1 ? "png"  : "svg"  );

  if ( $vis_format == 1 ) {
    $grid->to_png ( pngout => "$EP::Config::h->{TMP}/$vis_output_filename" );
  } else {
    $grid->to_svg ( toFile => "$EP::Config::h->{TMP}/$svg_output" );
  }

  $action->{action_metadata}->{clustering_url} = "$EP::Config::h->{EPNG_USRDATA_URL}/" . $dataset->folder->location . "/$action->{action_metadata}->{clustering}";

  return ( $vis_output_filename,
	   $grid->getGridWidth(1),    # nothing returned if not explicitly called with 1
	   $grid->getGridHeight(1) );
}

1;
